package com.gitee.wsl.struct.grammar.parser.tokenizer

import com.gitee.wsl.struct.grammar.annotations.ExperimentalParsusApi
import com.gitee.wsl.struct.grammar.parser.token.Token
import com.gitee.wsl.struct.grammar.parser.token.TokenMatch
import com.gitee.wsl.struct.grammar.parser.trace.TokenMatchingTrace

@OptIn(ExperimentalParsusApi::class)
interface Tokenizer {
    val input: String

    fun reset(input: String, traceTokenMatching: Boolean)

    fun getTokenMatchingTrace(): TokenMatchingTrace?
    fun findContextFreeMatch(fromIndex: Int): TokenMatch?
    fun findMatchOf(fromIndex: Int, targetToken: Token): TokenMatch?
}
